/**
  *    ------------------------------------------------------------------------------
  *                                                              
  *            NIKHEF - National Institute for Subatomic Physics 
  *  
  *                        Electronics Department                
  *                                                              
  *  ----------------------------------------------------------------------------
  *  @class wupper driver
  *  
  *  
  *  @author      Andrea Borga    (andrea.borga@nikhef.nl)<br>
  *               Frans Schreuder (frans.schreuder@nikhef.nl)<br>
  *               Markus Joos<br>
  *               Jos Vermeulen<br>
  *               Oussama el Kharraz Alami<br>
  *  
  *  
  *  @date        08/09/2015    created
  *  
  *  @version     1.0
  *  
  *  @brief Original version (RobinNP driver) by Barry Green, Will Panduro (RHUL),
  *  Gordon Crone (UCL), Markus Joos (CERN)
  *  Adapted for WUPPER by Jos Vermeulen (Nikhef), Jan. 2015"); 
  * 
  *  @detail
  *  
  *  ----------------------------------------------------------------------------
  *  @TODO
  *   
  *  
  *  ------------------------------------------------------------------------------
  *  Wupper
  *  
  *  \copyright GNU LGPL License
  *  Copyright (c) Nikhef, Amsterdam, All rights reserved. <br>
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Lesser General Public
  *  License as published by the Free Software Foundation; either
  *  version 3.0 of the License, or (at your option) any later version.
  *  This library is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  *  Lesser General Public License for more details.<br>
  *  You should have received a copy of the GNU Lesser General Public
  *  License along with this library.
  */

/************************************************************************/
/*									*/
/* File: wupper.c => wupper.c							        */
/*									*/
/* Driver for the WUPPER PCIe cards    				        */
/*									*/
/* Developed by J. Vermeulen (NIKHEF) by using code from robinnp.c	*/
/* Now maintained by M. Joos (CERN)					*/
/*									*/
/************ C 2023 - The software with that certain something *********/

// MJ: A note on return values:
// MJ: Where possible I have used standard UNIX error codes. E.g. "return(-ENODEV)"
// MJ: Inspration for the error code has been taked from "Linux device drivers, release 3"
// MJ: If the book was not clear I went to lxr.free-electrons.com and looked at what other drivers do.


// Clarifying terminology
// "card" refers to a WUPPER-7xx PCIe H/W module
// "device" refers to a PCIe endpoint
// Each WUPPER-712 card has two devices (some older WUPPER-7xx had just one)
// As Linux has a device perspective, this is the dominating terminology in this driver. 


// A bit of documentation for the resource locking mechanism
//
// global_locks[MAXCARDS]
// This is an array of 32bit intergegers. Each "int" represents the (max.) 32 lock bits of one 712 device. It contains the "or" of all resources that are currently locked on the respective device
//
// lock_tag
// This global variable is used to enumerate the locking actions. It is used to figure out to which process a locked bit belongs
// 
// lock_tags[MAXCARDS][MAXLOCKBITS]
// See wupper_common.h: Used to separate the locks of different WupperCard objects in the same thread
//
// lock_pid[MAXCARDS][MAXLOCKBITS];
// This array records the PID of the process that holds the respective lock


// Compatibility tracking
// We want to make sure the driver rejects to run on WUPPER cards that are equipped with incompatible F/W releases. 
// Therefore whenever a new (major) release of the F/W register model (RM) is made it has to be checked if 
// the driver uses registers that are subject to a change. In in order to facilitate this, the table below lists all registers that are
// accessed by the driver.
// Note: Registers in the PCIe config space are not listed
// | BAR | Offest | Name |
// |  2  | 0x0000 | REG_MAP_VERSION |
// |  2  | 0x0010 | BOARD_ID_TIMESTAMP |
// |  2  | 0x0040 | GIT_TAG |
// |  2  | 0x0050 | GIT_OMMIT_NUMBER |
// |  2  | 0x0060 | GIT_HASH |
// |  2  | 0x0080 | GENERIC_CONSTANTS |
// |  2  | 0x00A0 | CARD_TYPE |
// |  2  | 0x0190 | FIRMWARE_MODE |
// |  2  | 0x9360 | FPGA_DNA |
// |  2  | 0x9810 | CONFIG_FLASH_WR |
// |  2  | 0x9820 | CONFIG_FLASH_RD |


#include <linux/version.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/io.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/cdev.h>
#include <linux/pagemap.h>
#include <linux/page-flags.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/time.h>
#include <linux/delay.h>
#include <linux/spinlock.h>       //For the spin-lock 
#include <linux/seq_file.h>       //CS9

#include "wupper_common.h"
#include "regmap/regmap-struct.h"

#define ADD_XVC

#ifdef ADD_XVC
//Start of add on for ADD_XVC
#include <linux/mod_devicetable.h>
#include <linux/kdev_t.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/device.h>
#include "wupper/tdaq_drivers.h"
#include "wupper/xvc_pcie_driver.h"
#include "wupper/xvc_pcie_user_config.h"
//End of add on for ADD_XVC
#endif


/***********/
/*Constants*/
/***********/
#define PROC_MAX_CHARS         0x10000      //The max. length of the output of /proc/wupper
#define PCI_VENDOR_ID_WUPPER_FW   0x10ee       //Note: The different H/W types (709, 710, 711 and 712) do not have specific F/W files
                                            //      Therefore the device ID refers to the F/W release, not to the type of PCIe device
#define PCI_DEVICE_ID_WUPPER_FW1  0x7038
#define PCI_DEVICE_ID_WUPPER_FW2  0x7039

#define PCI_VENDOR_ID_CERN_FW  0x10dc
#define PCI_DEVICE_ID_CERN_FW1 0x0427
#define PCI_DEVICE_ID_CERN_FW2 0x0428

#define FIRSTMINOR             0
#define MAXMSIX		       8            // Max. number of interrupts (MSI-X) per device


/************/
/*Prototypes*/
/************/
static int wupper_init(void);
static int wupper_Probe(struct pci_dev*, const struct pci_device_id*);
static ssize_t wupper_write_procmem(struct file *file, const char *buffer, size_t count, loff_t *startOffset);
static void wupper_exit(void);
static void wupper_Remove(struct pci_dev*);
int wupper_mmap(struct file*, struct vm_area_struct*);
static long wupper_ioctl(struct file *file, u_int cmd, u_long arg);
int wupper_open(struct inode*, struct file*);
int wupper_Release(struct inode*, struct file*);
void wupper_vmclose(struct vm_area_struct*);
int wupper_proc_open(struct inode *inode, struct file *file);
int wupper_proc_show(struct seq_file *sfile, void *p);

#ifdef ADD_XVC
//Start of add on for ADD_XVC
static int xil_xvc_init(void);
static void xil_xvc_cleanup(void);
static int xil_xvc_probe(struct pci_dev *dev);
static void xil_xvc_remove(struct pci_dev *dev);
//End of add on for ADD_XVC
#endif


/************/
/*Structures*/
/************/
static struct pci_device_id WUPPER_IDs[] =
{
  { PCI_DEVICE(PCI_VENDOR_ID_WUPPER_FW, PCI_DEVICE_ID_WUPPER_FW1) },
  { PCI_DEVICE(PCI_VENDOR_ID_WUPPER_FW, PCI_DEVICE_ID_WUPPER_FW2) },
  { PCI_DEVICE(PCI_VENDOR_ID_CERN_FW, PCI_DEVICE_ID_CERN_FW1) },
  { PCI_DEVICE(PCI_VENDOR_ID_CERN_FW, PCI_DEVICE_ID_CERN_FW2) },
  { 0, },
};

struct file_operations fops =
{
  .owner          = THIS_MODULE,
  .mmap           = wupper_mmap,
  .unlocked_ioctl = wupper_ioctl,
  .open           = wupper_open,
  .release        = wupper_Release,
};


//CS9
//Inspiration taken from: https://stackoverflow.com/questions/64931555/how-to-fix-error-passing-argument-4-of-proc-create-from-incompatible-pointer
//===========================================================
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
static const struct proc_ops wupper_proc_file_ops = 
{
  .proc_open  = wupper_proc_open,
  .proc_write = wupper_write_procmem,
  .proc_read  = seq_read,    
  .proc_lseek = seq_lseek	
};
#else
static struct file_operations wupper_proc_file_ops = 
{
  .owner   = THIS_MODULE,
  .open    = wupper_proc_open,
  .write   = wupper_write_procmem,
  .read    = seq_read,
  .llseek  = seq_lseek,
  .release = single_release
};
#endif
//==============================================================





// needed by pci_register_driver fcall
static struct pci_driver wupper_PCI_driver =
{
  .name     = "wupper",
  .id_table = WUPPER_IDs,
  .probe    = wupper_Probe,
  .remove   = wupper_Remove,
};

// memory handler functions used by mmap
static struct vm_operations_struct wupper_vm_ops =
{
  .close =  wupper_vmclose,             // mmap-close
};

struct irqInfo_struct
{
  int interrupt;
  int device;
};


/*********/
/*Globals*/
/*********/
char *devName = "wupper";  //the device name as it will appear in /proc/devices
static char *proc_read_text;
static int debug = 0, errorlog = 1, autoswap = 0, deviceNumberIndex = 0;
int devicesFound = 0, interruptCount = 0, cardsignored = 0;
int msixblock = MAXMSIX, irqFlag[MAXCARDS][MAXMSIX] = {{0}}, msixStatus[MAXCARDS];
int irqMasked[MAXCARDS][MAXMSIX];
u_int irqCount[MAXCARDS][MAXMSIX];
u_int cdmap[MAXCARDS][2];
uint32_t* msixBar[MAXCARDS], msixPbaOffset[MAXCARDS];  //MJ: msixPbaOffset is only used in debug commands. Is it needed in the driver? What is it for?
card_params_t devices[MAXCARDS];
struct cdev *wupper_cdev;
dev_t first_dev;
static struct irqInfo_struct irqInfo[MAXCARDS][MAXMSIX];

static u_int global_locks[MAXCARDS];
u_int lock_tag = 1, lock_tags[MAXCARDS][MAXLOCKBITS], lock_pid[MAXCARDS][MAXLOCKBITS];
static u_long lock_irq_flags;

static DECLARE_WAIT_QUEUE_HEAD(waitQueue);
static DEFINE_MUTEX(procMutex);
static DEFINE_SPINLOCK(lock_lock);    //The spinlock for the resource locking

module_init(wupper_init);
module_exit(wupper_exit);

MODULE_DESCRIPTION("WUPPER driver");
MODULE_AUTHOR("Jos Vermeulen (Nikhef) and Markus Joos (CERN)");
MODULE_LICENSE("Dual BSD/GPL");
//MODULE_DEVICE_TABLE(pci, WUPPER_IDs); //Disabled by MJ in order to prevent the driver from auto loading. The driver will be loaded by /etc/init.d/drivers_wupper

MODULE_PARM_DESC(msixblock, "size of MSI-X block to enable. Maximum value = MAXMSIX ( = 8)");
module_param(msixblock, int, S_IRUGO);

MODULE_PARM_DESC(debug, "1 = enable debugging   0 = disable debugging");
module_param (debug, int, S_IRUGO | S_IWUSR);

MODULE_PARM_DESC(errorlog, "1 = enable error logging   0 = disable error logging");
module_param (errorlog, int, S_IRUGO | S_IWUSR);

MODULE_PARM_DESC(autoswap, "1 = enable reordering of 0x7038/0x7039 and 0x427/0x428   0 = disable reordering of 0x7038/0x7039 and 0x427/0x428");
module_param (autoswap, int, S_IRUGO | S_IWUSR);

struct msix_entry msixTable[MAXCARDS][MAXMSIX];

#ifdef ADD_XVC
//Start of add on for ADD_XVC
static bool xvcFlag[MAXCARDS];
/* from xvc_pcie_driver.c */
#define VALID_OFFSET(a) (a < 0x1000 && a >= 0x100)

#define VSEC_ID_MASK        0x0FFFF  // bits 15:0
#define VSEC_REV_MASK       0xF0000  // bits 19:16
#define VSEC_REV_SHIFT      16
#define NEXT_CAP_SHIFT      20
#define CHAR_DEVICES_MAX    CFG_DEVICES_MAX * USER_CONFIG_COUNT
#define DEBUG_ID_REG_OFFSET (xvc_offsets[algo->type].debug_id_reg_offset)
#define LENGTH_REG_OFFSET   (xvc_offsets[algo->type].length_reg_offset)
#define TMS_REG_OFFSET      (xvc_offsets[algo->type].tms_reg_offset)
#define TDI_REG_OFFSET      (xvc_offsets[algo->type].tdi_reg_offset)
#define TDO_REG_OFFSET      (xvc_offsets[algo->type].tdo_reg_offset)
#define CONTROL_REG_OFFSET  (xvc_offsets[algo->type].control_reg_offset)

static struct class *xvc_dev_class;
static const size_t CFG_DEVICES_MAX = 10;
static struct xil_xvc_char *xil_xvc_devices = NULL;
static long setup_xvc_algo(struct xil_xvc_char *xvc_char, const struct pcie_user_config *user_config);
long char_ctrl_ioctl(struct file *file_p, unsigned int cmd, unsigned long arg);

static int xvc_ioc_dev_region_status = -1;
static dev_t xvc_ioc_dev_region;
static struct cdev xvc_char_ioc_dev;

static dev_t get_device_number(dev_t region, int offset) {
  return MKDEV(MAJOR(region), MINOR(region) + offset);
}

static struct file_operations xil_xvc_ioc_ops = {
  .owner = THIS_MODULE,
  .unlocked_ioctl = char_ctrl_ioctl
};

struct xil_xvc_char {
  struct pci_dev *pci_dev;
  struct xvc_algo_t xvc_algo;
  struct pcie_user_config *user_config;
};

struct xvc_offsets {
  size_t debug_id_reg_offset;
  size_t length_reg_offset;
  size_t tms_reg_offset;
  size_t tdi_reg_offset;
  size_t tdo_reg_offset;
  size_t control_reg_offset;
};

static const struct xvc_offsets xvc_offsets[] = {
  {0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, // XVC_ALGO_NULL
  {0x08, 0x0C, 0x10, 0x14, 0x14, 0x18}, // XVC_ALGO_CFG
  {0x00, 0x00, 0x04, 0x08, 0x0C, 0x10}  // XVC_ALGO_BAR
};

//End of add on for ADD_XVC
#endif


/*******************************************************/
int wupper_proc_open(struct inode *inode, struct file *file) 
/*******************************************************/
{
  int ret;
  
  ret = single_open(file, wupper_proc_show, NULL);
  if (ret)    
    kerror(("wupper(wupper_proc_open): Error %d received from single_open\n", ret))
    
  return(ret);
}


/***********************/
static int wupper_init(void)
/***********************/
{
  int stat, deviceNumber, lbit, interrupt, loop;
  struct proc_dir_entry *procDir;

  proc_read_text = (char *)kmalloc(PROC_MAX_CHARS, GFP_KERNEL);
  if (proc_read_text == NULL)
  {
    kerror(("wupper(wupper_init): error from kmalloc\n"))  
    return(-ENOMEM);
  }
  
#ifdef ADD_XVC
  //Start of add on for ADD_XVC
  stat = xil_xvc_init();
  if(stat != 0) 
  {
    kerror(("wupper(wupper_init): Error %d received from xil_xvc_init\n", stat))
    return stat;
  }
  //End of add on for ADD_XVC
#endif  
  
  for (loop = 0; loop < MAXCARDS; loop++)
  {
    cdmap[loop][0] = 0;
    cdmap[loop][1] = 0;
  }

  for (deviceNumber = 0; deviceNumber < MAXCARDS; deviceNumber++)
  {
#ifdef ADD_XVC
    xvcFlag[deviceNumber] = false;
#endif
    devices[deviceNumber].pciDevice = NULL;
    for (interrupt = 0; interrupt < MAXMSIX; interrupt++)
    {
      kdebug(("wupper(wupper_init): initializing IRQ values for interrupt %d of device %d\n", interrupt, deviceNumber))
      irqCount[deviceNumber][interrupt] = 0;
      irqMasked[deviceNumber][interrupt] = 1;
      irqFlag[deviceNumber][interrupt] = 0;
    }
    global_locks[deviceNumber] = 0;

    for (lbit = 0; lbit < MAXLOCKBITS; lbit++)
    {
      lock_pid[deviceNumber][lbit] = 0;  
      lock_tags[deviceNumber][lbit] = 0;  
    }
  }

  if (msixblock > MAXMSIX)
  {
    kerror(("wupper(wupper_init):msixblock > MAXMSIX - setting to max (%d)\n", MAXMSIX))
    msixblock = MAXMSIX;  //MJ: We could be less friendly and cause the driver installation to fail if the user spceifies and out-of-bounds value
  }

  kdebug(("wupper(wupper_init): registering PCIDriver \n"))
  stat = pci_register_driver(&wupper_PCI_driver);
  if (stat < 0)
  {
    kerror(("wupper(wupper_init): Status %d from pci_register_driver\n", stat))
    return stat;
  }

  procDir = proc_create(devName, 0644, NULL, &wupper_proc_file_ops);
  if (procDir == NULL)
  {
    kerror(("wupper(wupper_init): error from call to proc_create\n"))
    return(-ENOMEM);     //MJ: The error code is a bit random. I have also seen -EINVAL in other driver. Many drivers just return -1
  }

  stat = alloc_chrdev_region(&first_dev, FIRSTMINOR, MAXCARDS, devName);
  if (stat == 0)
  {
    wupper_cdev = cdev_alloc();
    wupper_cdev->ops = &fops;
    wupper_cdev->owner = THIS_MODULE;

    for (deviceNumber = 0; deviceNumber < MAXCARDS; deviceNumber++)
    {
      kdebug(("wupper(wupper_init): calling cdev_add for device %d\n", deviceNumber))
      stat = cdev_add(wupper_cdev, first_dev + deviceNumber, 1);
      if (stat != 0)
      {
        kerror(("wupper(wupper_init): cdev_add failed at device %d, driver will not load\n", deviceNumber))  
        unregister_chrdev_region(first_dev, MAXCARDS);
        pci_unregister_driver(&wupper_PCI_driver);
        return(stat);
      }
    }
  }
  else
  {
    kerror(("wupper_init: registering WUPPER driver failed.\n"))
    pci_unregister_driver(&wupper_PCI_driver);
    return(stat);
  }

  kerror(("wupper(wupper_init): WUPPER driver loaded, found %d device(s)\n", devicesFound))   //MJ: this is not an error but it is nice to have this information always in the log file 
  return 0;
}


/************************/
static void wupper_exit(void)
/************************/
{
  remove_proc_entry(devName, NULL /* parent dir */);
  kdebug(("wupper(wupper_exit): unregister device\n"))
  
  unregister_chrdev_region(first_dev, MAXCARDS);
  kdebug(("wupper(wupper_exit: unregister driver\n"))
  
  pci_unregister_driver(&wupper_PCI_driver);
  cdev_del(wupper_cdev);
  kfree(proc_read_text);

#ifdef ADD_XVC
  //Start of add on for ADD_XVC
  xil_xvc_cleanup();
  //End of add on for ADD_XVC
#endif

  kdebug(("wupper(wupper_exit): driver removed\n"))
}


/***********************************************/
static irqreturn_t irqHandler(int irq, void *dev)
/***********************************************/
{
  struct irqInfo_struct *info;

  info = (struct irqInfo_struct*) dev;

  kdebug(("wupper(irqHandler, pid=%d): Interrupt %d received from device %d\n", current->pid, info->interrupt, info->device))

  irqCount[info->device][info->interrupt] += 1;
  irqFlag[info->device][info->interrupt] = 1;
  wake_up_interruptible(&waitQueue);      //MJ: would it have any performance advantages if we used one wait queue per device?
  return(IRQ_HANDLED);
}


/***********************************************************************/
static int wupper_Probe(struct pci_dev *dev, const struct pci_device_id *id)
/***********************************************************************/
{
  static int first_did = 0;
  int deviceNumber, ret, bufferNumber, interrupt, msixCapOffset, msixData, msixBarNumber, msixTableOffset, msixLength;
  uint32_t msixAddress;    
  u_long ldata;
  wuppercard_bar2_regs_t *rm;

  kdebug(("wupper(wupper_Probe): DID = 0x%08x, VID = 0x%08x\n", dev->device, dev->vendor))

  //Note: For the proper functioning of the wupper tools it is important that the wupper devices (remember: each PCIe module has got one or two) are properly ordered
  //      The device 7038 has to be before 7039. On some (maybe all?) PCs the 712 modules are presented to the driver in the wrong order.
  //      the code below remaps the devices.

  deviceNumber = deviceNumberIndex;
  kdebug(("wupper(wupper_Probe): deviceNumber = %d\n", deviceNumber))

  //In pc-tbed-felix-05 we have two modules that both only provide one end-pint with DID=7038. In such a case the auto-swap causes havoc
  if (first_did == 0)
  {
    if (dev->device == PCI_DEVICE_ID_WUPPER_FW2 || dev->device ==  PCI_DEVICE_ID_CERN_FW2)
    {
      autoswap = 1;
      kdebug(("wupper(wupper_Probe): auto swapping enabled\n"))   
    }      
    first_did = 1;
  }

  if (autoswap)
  {
    if (dev->device == PCI_DEVICE_ID_WUPPER_FW2 && !(deviceNumberIndex & 1))  //7039 and even index
    {
      kdebug(("wupper(wupper_Probe): auto remap 1\n"))   
      deviceNumber++;
    }

    if (dev->device == PCI_DEVICE_ID_WUPPER_FW1 && (deviceNumberIndex & 1))   //7038 and odd index
    {
      kdebug(("wupper(wupper_Probe): auto remap 2\n"))  
      deviceNumber--;
    }    

    if (dev->device == PCI_DEVICE_ID_CERN_FW2 && !(deviceNumberIndex & 1))  //428 and even index
    {
      kdebug(("wupper(wupper_Probe): auto remap 3\n"))   
      deviceNumber++;
    }

    if (dev->device == PCI_DEVICE_ID_CERN_FW1 && (deviceNumberIndex & 1))   //427 and odd index
    {
      kdebug(("wupper(wupper_Probe): auto remap 4\n"))   
      deviceNumber--;
    }   
    kdebug(("wupper(wupper_Probe): new deviceNumber = %d\n", deviceNumber))   
  }

  deviceNumberIndex++;

  if (deviceNumber < MAXCARDS)
  {
    kdebug(("wupper(wupper_Probe): Initialising logical device nr %d (counting from 0)\n", devicesFound))
    ret = pci_enable_device(dev); //according to https://www.kernel.org/doc/Documentation/PCI/pci.txt this function can fail
    if (ret)
    {
      kerror(("wupper(wupper_Probe): Error %d received from pci_enable_device. \n", ret))
      //Maybe the function pcibios_strerror() can be helpful to translate the error code into a string
      deviceNumberIndex--;
      return(-EINVAL);
    }   
  
    if (dev->current_state == PCI_D0)
      kdebug(("wupper(wupper_Probe): Power state is D0\n"))
    else
    {
      kerror(("wupper(wupper_Probe): Power state is not D0 but %d. Refusing to manage this device\n", dev->current_state))
      cardsignored++;
      pci_disable_device(dev);
      deviceNumberIndex--;
      return(-EINVAL);
    }

    devicesFound++;
    devices[deviceNumber].pciDevice = dev;
  }
  else
  {
    kerror(("wupper(wupper_Probe): Too many devices present, only %d is allowed\n", MAXCARDS))
    return(-EINVAL);
  }

  kdebug(("wupper(wupper_Probe): Reading configuration space for device %d :\n", deviceNumber))
  devices[deviceNumber].baseAddressBAR0 = pci_resource_start(dev, 0);
  devices[deviceNumber].sizeBAR0        = pci_resource_len(dev, 0);
  devices[deviceNumber].baseAddressBAR1 = pci_resource_start(dev, 1);
  devices[deviceNumber].sizeBAR1        = pci_resource_len(dev, 1);
  devices[deviceNumber].baseAddressBAR2 = pci_resource_start(dev, 2);
  devices[deviceNumber].sizeBAR2        = pci_resource_len(dev, 2);

  kdebug(("wupper(wupper_Probe): BAR0 start 0x%lx, end 0x%x, size 0x%lx \n", devices[deviceNumber].baseAddressBAR0, (u_int)pci_resource_end(dev, 0), devices[deviceNumber].sizeBAR0))
  kdebug(("wupper(wupper_Probe): BAR1 start 0x%lx, end 0x%x, size 0x%lx \n", devices[deviceNumber].baseAddressBAR1, (u_int)pci_resource_end(dev, 1), devices[deviceNumber].sizeBAR1))
  kdebug(("wupper(wupper_Probe): BAR2 start 0x%lx, end 0x%x, size 0x%lx \n", devices[deviceNumber].baseAddressBAR2, (u_int)pci_resource_end(dev, 2), devices[deviceNumber].sizeBAR2))

  msixCapOffset = pci_find_capability(dev, PCI_CAP_ID_MSIX);
  if (msixCapOffset == 0)
  {
    // module may not have wupper hardware loaded
    kerror(("wupper(wupper_Probe): Failed to map MSI-X BAR for device %d\n", deviceNumber))
    msixBar[deviceNumber] = NULL;
    return(-ENODEV);
  }

  // MSI-X table offset
  pci_read_config_dword(dev, msixCapOffset + PCI_MSIX_TABLE, &msixData);
  msixBarNumber = msixData & PCI_MSIX_TABLE_BIR;
  msixTableOffset = msixData & PCI_MSIX_TABLE_OFFSET;
  kdebug(("wupper(wupper_Probe): MSIX Vector table BAR %d, offset %08x\n", msixBarNumber, msixTableOffset))
  // MSI-X Pending Bit Array offset
  pci_read_config_dword(dev, msixCapOffset + PCI_MSIX_PBA, &msixData);
  msixBarNumber = msixData & PCI_MSIX_PBA_BIR;
  msixPbaOffset[deviceNumber] = msixData & PCI_MSIX_PBA_OFFSET;
  kdebug(("wupper(wupper_Probe): MSIX PBA: BAR %d, offset %08x\n", msixBarNumber, msixPbaOffset[deviceNumber]))
  msixAddress = pci_resource_start(dev, msixBarNumber);
  msixLength = pci_resource_len(dev, msixBarNumber);
    
  kdebug(("wupper(wupper_Probe): msixAddress = 0x%08x, msixLength = %d\n", msixAddress, msixLength))
  
  //CS9
  //===========================================
  //Inspiration taken from https://github.com/walkco/rts5227/pull/2  
  
  #if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
  msixBar[deviceNumber] = ioremap(msixAddress, msixLength);  
  #else
  msixBar[deviceNumber] = ioremap_nocache(msixAddress, msixLength);
  #endif
  //===========================================

  if (msixBar[deviceNumber] == NULL)
  {
    kerror(("wupper(wupper_Probe): FAILED to map MSI-X BAR for device %d\n", deviceNumber))
    return(-EINVAL);
  }

  if (debug)
  {
    bufferNumber = msixTableOffset / sizeof(uint32_t);
    for (interrupt = 0; interrupt < MAXMSIX; interrupt++)
    {
      kdebug(("wupper(wupper_Probe): MSI-X table[%d] %08x %08x  %08x  %08x\n", interrupt, msixBar[deviceNumber][bufferNumber], msixBar[deviceNumber][bufferNumber + 1], msixBar[deviceNumber][bufferNumber + 2], msixBar[deviceNumber][bufferNumber + 3]))
      bufferNumber += 4;
    }

    if (msixPbaOffset[deviceNumber] + 3 * sizeof(uint32_t) < msixLength)
    {
      kdebug(("wupper(wupper_Probe): MSI-X PBA      %08x %08x  %08x  %08x\n",
              msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t)],
              msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t) + 1],
              msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t) + 2],
              msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t) + 3]))
    }
    else
      kerror(("wupper(wupper_Probe): PBA offset 0x%x is outside of BAR%d, length=0x%x \n", msixPbaOffset[deviceNumber], msixBarNumber, msixLength))
  }

  // setup interrupts
  if (msixblock < MAXMSIX)
    kerror(("wupper(wupper_Probe): WARNING: msixblock(%d) < MAXMSIX(%d). I hope you know what you are doing\n", msixblock, MAXMSIX))

  for (interrupt = 0; interrupt < msixblock; interrupt++)
  {
    msixTable[deviceNumber][interrupt].entry = interrupt;  //MJ: If (msixblock < MAXMSIX) some elements of the array will not be initialized (which may not matter)
    kdebug(("wupper(wupper_Probe): filling interrupt table for interrupt %d, deviceNumber %d\n", interrupt, deviceNumber))
    kdebug(("wupper(wupper_Probe): entry in table %d\n", msixTable[deviceNumber][interrupt].entry))
  }

#if LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0)
  msixStatus[deviceNumber] = pci_enable_msix(dev, msixTable[deviceNumber], msixblock);  
  if (msixStatus[deviceNumber] != 0)
#else  
  msixStatus[deviceNumber] = pci_enable_msix_range(dev, msixTable[deviceNumber], 1, msixblock);  
  if (msixStatus[deviceNumber] != msixblock)
#endif
  {
    kerror(("wupper(wupper_Probe): Failed to enable MSI-X interrupt block for device %d, pci_enable_msix(_range) returned %d\n", deviceNumber, msixStatus[deviceNumber]))
    return(-EINVAL);
  }

  if (debug)
  {
    kdebug(("wupper(wupper_Probe): msix address %08x, length %4x\n", msixAddress, msixLength))
    bufferNumber = msixTableOffset / sizeof(uint32_t);
    for (interrupt = 0; interrupt < msixblock; interrupt++)
    {
      kdebug(("wupper(wupper_Probe): MSI-X table[%d] %08x %08x  %08x  %08x\n", interrupt, msixBar[deviceNumber][bufferNumber], msixBar[deviceNumber][bufferNumber+1], msixBar[deviceNumber][bufferNumber+2], msixBar[deviceNumber][bufferNumber+3]))
      bufferNumber += 4;
    }
    if (msixPbaOffset[deviceNumber] + 3 * sizeof(uint32_t) < msixLength)
      kdebug(("wupper(wupper_Probe): MSI-X PBA %08x \n", msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t)]))
    else
      kerror(("wupper(wupper_Probe): PBA offset 0x%x is outside of BAR%d, length=0x%x \n", msixPbaOffset[deviceNumber], msixBarNumber, msixLength))
  }

  for (interrupt = 0; interrupt < msixblock; interrupt++)
  {
    kdebug(("wupper(wupper_Probe): Trying to register IRQ vector %d\n", msixTable[deviceNumber][interrupt].vector))

    irqInfo[deviceNumber][interrupt].interrupt = interrupt;     //MJ: If we have two devices then two elements of the array will have the same value (interrupt). Does that matter?
    irqInfo[deviceNumber][interrupt].device    = deviceNumber;    
    ret = request_irq(msixTable[deviceNumber][interrupt].vector, irqHandler, 0, devName, &irqInfo[deviceNumber][interrupt]);
    if (ret != 0)
    {
      kerror(("wupper(wupper_Probe): Failed to register interrupt handler for MSI %d\n", interrupt))
      return(-EINVAL);
    }

    kdebug(("wupper(wupper_Probe): disable the interrupt. deviceNumber = %d, interrupt = %d, vector = %d\n", deviceNumber, interrupt, msixTable[deviceNumber][interrupt].vector)) 
    disable_irq(msixTable[deviceNumber][interrupt].vector);  //let the user enable the interrupt
  }

  // do reset
  if (debug)
  {
    kdebug(("wupper(wupper_Probe): msix address %08x, length %4x\n", msixAddress, msixLength))
    bufferNumber = msixTableOffset / sizeof(uint32_t);
    for (interrupt = 0; interrupt < msixblock; interrupt++)
    {
      kdebug(("wupper(wupper_Probe): MSI-X table[%d] %08x %08x  %08x  %08x\n", interrupt, msixBar[deviceNumber][bufferNumber], msixBar[deviceNumber][bufferNumber+1], msixBar[deviceNumber][bufferNumber+2], msixBar[deviceNumber][bufferNumber+3]))
      bufferNumber += 4;
    }

    if (msixPbaOffset[deviceNumber] + 3 * sizeof(uint32_t) < msixLength)
    {
        kdebug(("wupper(wupper_Probe): MSI-X PBA %08x %08x  %08x  %08x\n",
                msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t)],
                msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t) + 1],
                msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t) + 2],
                msixBar[deviceNumber][msixPbaOffset[deviceNumber] / sizeof(uint32_t) + 3]))
    }
    else
      kerror(("wupper(wupper_Probe): PBA offset 0x%x is outside of BAR%d, length=0x%x \n", msixPbaOffset[deviceNumber], msixBarNumber, msixLength))
  }
  
  //Identify the card type. The CARD_TYPE register is at offset 0xA0 of BAR2
  kerror(("BAR2 Address: %lX, BAR2 Size: %liM\n",devices[deviceNumber].baseAddressBAR2,devices[deviceNumber].sizeBAR2/(1024*1024)));
  
  //CS9
  //===========================================
  //Inspiration taken from https://github.com/walkco/rts5227/pull/2  
  
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
  rm = ioremap(devices[deviceNumber].baseAddressBAR2, devices[deviceNumber].sizeBAR2);
#else
  rm = ioremap_nocache(devices[deviceNumber].baseAddressBAR2, devices[deviceNumber].sizeBAR2);
#endif
  //===========================================
  ldata  = rm->CARD_TYPE;
  kdebug(("wupper(wupper_Probe): CARD_ID of device %d = 0x%016lx\n", deviceNumber, ldata));
  cdmap[deviceNumber][0] = ldata & 0xfff;  //card type
  
  if (dev->device == PCI_DEVICE_ID_WUPPER_FW1 || dev->device == PCI_DEVICE_ID_CERN_FW1)
  {
    cdmap[deviceNumber][1] = 0;               //device of card
    kdebug(("wupper(wupper_Probe): Relative device of device %d = 0\n", deviceNumber));
  } 

  if (dev->device == PCI_DEVICE_ID_WUPPER_FW2 || dev->device == PCI_DEVICE_ID_CERN_FW2)
  {
    cdmap[deviceNumber][1] = 1;               //device of card
    kdebug(("wupper(wupper_Probe): Relative device of device %d = 1\n", deviceNumber));
  }
  
#ifdef ADD_XVC
  //Start of add on for ADD_XVC
  if (dev->device == PCI_DEVICE_ID_CERN_FW1)
  {
    ret = xil_xvc_probe(dev);
    if(ret != 0)
    {
      kerror(("wupper(wupper_Probe): Error %d received from xil_xvc_probe\n", ret))  //MJ: Just a warning. The F/W does not support XVC
      xvcFlag[deviceNumber] = false;
    }
    else
    {
       xvcFlag[deviceNumber] = true;
    }
  }
  //End of add on for ADD_XVC
#endif
  
  return(0);
}


/*****************************************/
static void wupper_Remove(struct pci_dev *dev)
/*****************************************/
{
  int deviceNumber, interrupt;

  kdebug(("wupper(wupper_Remove):  called\n"))
  for(deviceNumber = 0; deviceNumber < MAXCARDS; deviceNumber++)
  {
#ifdef ADD_XVC
    //Start of add on for ADD_XVC
    if (xvcFlag[deviceNumber])
    {
      xil_xvc_remove(dev);
    }
    //End of add on for ADD_XVC
#endif
    if (devices[deviceNumber].pciDevice == dev)
    {
      kdebug(("wupper(wupper_Remove): for device %d\n", deviceNumber))
      devices[deviceNumber].pciDevice = NULL;
      devicesFound--;
      deviceNumberIndex--;  //MJ: this is dangerous if devices are removed (and added) in random oder (reference: script pcie_hotplug_remove.sh)
      kdebug(("wupper(wupper_Remove): lowering  deviceNumberIndex to %d\n", deviceNumberIndex))

#if LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0)
        if (msixStatus[deviceNumber] == 0)
#else
        if (msixStatus[deviceNumber] == msixblock)
#endif
      {
        for (interrupt = 0; interrupt < msixblock; interrupt++)
        {
          kdebug(("wupper(wupper_Remove): unregistering interrupt %d, vector %d\n", interrupt, msixTable[deviceNumber][interrupt].vector))
          free_irq(msixTable[deviceNumber][interrupt].vector, &irqInfo[deviceNumber][interrupt]);
        }
      }
      pci_disable_msix(dev);
    }
  }
}


/************************************************/
int wupper_open(struct inode *ino, struct file *file)
/************************************************/
{
  card_params_t *pdata;

  kdebug(("wupper(wupper_open): called for PID = %d\n", current->pid))
  pdata = (card_params_t *)kmalloc(sizeof(card_params_t), GFP_KERNEL);
  if (pdata == NULL)
  {
    kerror(("wupper(wupper_open): error from kmalloc\n"))
    return(-ENOMEM);
  }

  pdata->slot = 99;  //99 means: No process has connected itself to the respective device
  file->private_data = (char *)pdata;
  return(0);
}


/***************************************************/
int wupper_Release(struct inode *ino, struct file *file)
/***************************************************/
{
  card_params_t *pdata;
  int lbit;

  //MJ lock: Do we get into trouble if a PID links to both devices???
  kdebug(("wupper(wupper_release): called for PID = %d\n", current->pid))

  pdata = (card_params_t *)file->private_data;
  kdebug(("wupper(wupper_release): pdata->slot     = %d\n", pdata->slot))
  kdebug(("wupper(wupper_release): pdata->lock_tag = %d\n", pdata->lock_tag))
  kdebug(("wupper(wupper_release): current->pid    = %d\n", current->pid))

  //This function gets called when a process closes /dev/wupper. We should put here the garbage collection for the lock bits.
  //Orphaned locks must be released when the last WupperCard object of a process gets destroyed. 

  if(pdata->slot != 99)  //Only run this code fragment if a user process is attached to the device.
  {
    spin_lock_irqsave(&lock_lock, lock_irq_flags);   //Please do not disturb...
    kdebug(("wupper(wupper_release): Old global_locks[%d] = 0x%08x\n", pdata->slot, global_locks[pdata->slot]))
    for(lbit = 0; lbit < MAXLOCKBITS; lbit++)
    {
      kdebug(("wupper(wupper_release): lock_pid[%d][%d]  = %d, lock_tags[%d][%d] = %d\n", pdata->slot, lbit, lock_pid[pdata->slot][lbit], pdata->slot, lbit, lock_tags[pdata->slot][lbit]))

      if (lock_tags[pdata->slot][lbit] == pdata->lock_tag)
      {
        kdebug(("wupper(wupper_release): unregistering orphaned bit %d of device %d for PID %d\n", lbit, pdata->slot, current->pid))
        lock_pid[pdata->slot][lbit] = 0;
        lock_tags[pdata->slot][lbit] = 0;
        global_locks[pdata->slot] = global_locks[pdata->slot] & ~(1 << lbit);
      }
    }
    kdebug(("wupper(wupper_release): New global_locks[%d] = 0x%08x\n", pdata->slot, global_locks[pdata->slot]))
    spin_unlock_irqrestore(&lock_lock, lock_irq_flags);   
  }

  kfree(file->private_data);
  kdebug(("wupper(wupper_release): Resources of slot %d released\n", pdata->slot))
  return(0);
}


/************************************************/
int wupper_proc_show(struct seq_file *sfile, void *p)
/************************************************/
{
  static int merror;
  int interrupt, deviceIndex[MAXCARDS], index, buildMonth, buildDay, buildHour, buildMinute;
  u_int device, lbit, genConstants, git_commit_number, git_hash, firmware_mode, device_type;
  u_int buildDate, buildYear, buildRevision, buildRevision2;
  u_long fpga_dna;
  char rmbyte;
  
#ifdef ADD_XVC
  //Start of add on for ADD_XVC
  int index_used = 0;
  int d, u, i;
  const char *name;
  //End of add on for ADD_XVC
#endif

  kdebug(("wupper(wupper_proc_show): Creating text....\n"));

  merror = mutex_lock_interruptible(&procMutex);
  if (merror)
  {
    kdebug(("wupper(wupper_proc_show): mutex lock not OK. error = %d\n", merror))
    return(0);
  }
  kdebug(("wupper(wupper_proc_show): mutex lock OK\n"))

  index = 0;
  for (device = 0; device < devicesFound; device++)
  {
    while (devices[index].pciDevice == NULL) index++;  //Look for the next device

    if (index < MAXCARDS)
    {
      deviceIndex[device] = index;
      index++;
      kdebug(("wupper(wupper_proc_show): device %d has index %d\n", device, index))
    }
    else
    {
      kerror(("wupper(wupper_proc_show): Device indexing error\n"))
      return(0);
    }
  }

  kdebug(("wupper(wupper_proc_show): Creating text....\n"))
  
#ifdef ADD_XVC
  seq_printf(sfile, "WUPPER driver (ALMA9 ready) for FELIX release 4.14 (compatible with RM4 and RM5 F/W and XVC). Based on tag %s, wupper.c revision %s\n", CVSTAG, WUPPER_TAG);
#else  
  seq_printf(sfile, "WUPPER driver (ALMA9 ready) for FELIX release 4.14 (compatible with RM4 and RM5 F/W. XVC is not supported). Based on tag %s, wupper.c revision %s\n", CVSTAG, WUPPER_TAG);
#endif

  if(cardsignored)
    seq_printf(sfile, "ERROR: %d card(s) were ignored because of a problem with the power status\n", cardsignored);

  seq_printf(sfile, "\nDebug                         = %d\n", debug);
  seq_printf(sfile, "Number of devices detected    = %d\n\n", devicesFound);

  //First we show the global lock bits. This can be removed later
  seq_printf(sfile, "\nLocked resources\n");
  seq_printf(sfile, "      device | global_locks\n");
  seq_printf(sfile, "=============|=============\n");
  for (device = 0; device < devicesFound; device++)
    seq_printf(sfile, "           %d |   0x%08x\n", device, global_locks[device]);

  //And now the individual locks
  seq_printf(sfile, "\nLocked resources\n");
  seq_printf(sfile, "device | resource bit |     PID |  tag\n");
  seq_printf(sfile, "=======|==============|=========|=====\n");
  for (device = 0; device < devicesFound; device++)
  {
    for (lbit = 0; lbit < MAXLOCKBITS; lbit++)
      if (lock_pid[device][lbit] != 0)	
	seq_printf(sfile, "     %d |           %2d | %7d |%5d\n", device, lbit, lock_pid[device][lbit], lock_tags[device][lbit]); 
  }

  for (device = 0; device < devicesFound; device++)
  {
    // Addresses depend on firmware version
    
    //CS9
    //===========================================
    //Inspiration taken from https://github.com/walkco/rts5227/pull/2  
  
    #if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
    uint32_t *pp = ioremap(devices[device].baseAddressBAR2, devices[device].sizeBAR2);
    uint64_t *lpp = ioremap(devices[device].baseAddressBAR2, devices[device].sizeBAR2);
    #else
    uint32_t *pp = ioremap_nocache(devices[device].baseAddressBAR2, devices[device].sizeBAR2);
    uint64_t *lpp = ioremap_nocache(devices[device].baseAddressBAR2, devices[device].sizeBAR2);
    #endif
    //===========================================

    u_int regmap_version = pp[0x00];

    if ((regmap_version & 0xf00) != 0x200)
    {
      seq_printf(sfile, "\nError: Device %d does not have the required F/W. The regmap register contains 0x%08x\n", deviceIndex[device], regmap_version);
      seq_printf(sfile, "\nError: This version of the driver is for regmap 2.0\n");
      continue;
    }

    buildDate        = pp[0x10/4];  //0x10/4 = 16/4 = 4. 4*4 bytes = 16 = 0x10 = offset in BAR2
    buildYear        = pp[0x14/4];
    buildRevision    = pp[0x40/4];
    buildRevision2   = pp[0x44/4];
    genConstants           = pp[0x80/4];
    git_commit_number      = pp[0x50/4];
    git_hash               = pp[0x60/4];
    device_type            = pp[0xA0/4];

    buildMonth  = (buildDate >> 24) & 0xff;
    buildDay    = (buildDate >> 16) & 0xff;
    buildHour   = (buildDate >> 8) & 0xff;
    buildMinute = buildDate & 0xff;
    seq_printf(sfile, "\nDevice %d: (BAR0 = 0x%lx)\n", deviceIndex[device], devices[device].baseAddressBAR0);

    seq_printf(sfile, "Card type                   : WUPPER-%d\n", device_type);
    seq_printf(sfile, "Device type                 : 0x%04x\n", devices[device].pciDevice->device);

    fpga_dna  = lpp[0x9360/8]; // FELIX FPGA_DNA register
    seq_printf(sfile, "FPGA_DNA                    : 0x%016lx\n", fpga_dna);

    seq_printf(sfile, "Reg Map Version             : %d.%d\n", (regmap_version & 0xFF00) >> 8, regmap_version & 0x00FF);

    seq_printf(sfile, "GIT tag                     : ");

    rmbyte = buildRevision & 0xff;	  if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = (buildRevision >> 8) & 0xff;	  if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = (buildRevision >> 16) & 0xff;  if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = (buildRevision >> 24) & 0xff;  if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = buildRevision2 & 0xff;	  if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = (buildRevision2 >> 8) & 0xff;  if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = (buildRevision2 >> 16) & 0xff; if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    rmbyte = (buildRevision2 >> 24) & 0xff; if (rmbyte) seq_printf(sfile, "%c", rmbyte);
    seq_printf(sfile, "\n");

    seq_printf(sfile, "BUILD Date and time         : %x-%x-20%x at %02xh%02x\n", buildDay, buildMonth, buildYear, buildHour, buildMinute);
    seq_printf(sfile, "GIT commit number           : %d\n", git_commit_number);
    seq_printf(sfile, "GIT hash                    : 0x%08x\n", git_hash);

    firmware_mode = pp[0x190/4];
    firmware_mode &= 0xF;
	
    seq_printf(sfile, "Firmware mode               : ");
         if (firmware_mode == 0) seq_printf(sfile, "GBT\n");
    else if (firmware_mode == 1) seq_printf(sfile, "FULL\n");
    else if (firmware_mode == 2) seq_printf(sfile, "LTDB\n");
    else if (firmware_mode == 3) seq_printf(sfile, "FEI4\n");
    else if (firmware_mode == 4) seq_printf(sfile, "ITK PIXEL\n");
    else if (firmware_mode == 5) seq_printf(sfile, "ITK STRIP\n");
    else if (firmware_mode == 6) seq_printf(sfile, "FELIG\n");
    else if (firmware_mode == 7) seq_printf(sfile, "FULL mode emulator\n");
    else if (firmware_mode == 8) seq_printf(sfile, "FELIX_MROD\n");
    else if (firmware_mode == 9) seq_printf(sfile, "lpGBT\n");
    else if (firmware_mode == 10) seq_printf(sfile, "INTERLAKEN\n");
    else if (firmware_mode == 11) seq_printf(sfile, "FELIG_LPGBT\n");
    else if (firmware_mode == 12) seq_printf(sfile, "HGTD_LUMI\n");
    else if (firmware_mode == 13) seq_printf(sfile, "BCM_PRIME\n");
    else if (firmware_mode == 14) seq_printf(sfile, "FELIG_PIXEL\n");
    else                         seq_printf(sfile, "Unknown (firmware_mode = %d)\n", firmware_mode);

    seq_printf(sfile, "Number of descriptors       : %d\n", (genConstants & 0xff));
    seq_printf(sfile, "Number of interrupts        : %d\n", ((genConstants >> 8) & 0xff));
    seq_printf(sfile, "Interrupt name  |ToHost 0|ToHost 1|ToHost 2|ToHost 3|reserved|CR Xoff |BUSY    |TH full |\n");

    if (msixBar[deviceIndex[device]] != NULL)
    {
      seq_printf(sfile, "Interrupt count |");
      for (interrupt = 0; interrupt < msixblock; interrupt++)
        seq_printf(sfile, " %6d |", irqCount[deviceIndex[device]][interrupt]);

      seq_printf(sfile, "\nInterrupt flag  |");
      for (interrupt = 0; interrupt < msixblock; interrupt++)
        seq_printf(sfile, " %6d |", irqFlag[deviceIndex[device]][interrupt]);

      seq_printf(sfile, "\nInterrupt mask  |");
      for (interrupt = 0; interrupt < msixblock; interrupt++)
        seq_printf(sfile, " %6d |", irqMasked[deviceIndex[device]][interrupt]);

      seq_printf(sfile, "\nMSI-X PBA       %08x\n",  msixBar[deviceIndex[device]][msixPbaOffset[deviceIndex[device]] / sizeof(uint32_t)]);
      seq_printf(sfile, "\n");
    }
    else
    {
      seq_printf(sfile, "No MSI-X interrupts for device %d\n\n", device);
    }

#ifdef ADD_XVC
    //Start of add on for ADD_XVC
    seq_printf(sfile, "XVC:\n");
    for (d = 0; d < CFG_DEVICES_MAX; ++d) 
    {
      if (devices[device].pciDevice != NULL && xil_xvc_devices[d * USER_CONFIG_COUNT].pci_dev == devices[device].pciDevice) 
      {
	for (u = 0; u < USER_CONFIG_COUNT; ++u) 
	{
          i = d * USER_CONFIG_COUNT + u;

          name = xil_xvc_devices[i].user_config->name;
          if (name && name[0]) 
          {
            seq_printf(sfile, "Xilinx Virtual Cable (XVC) associated with /dev/xil_xvc/cfg_ioc%d_%s\n", d, name);
          }
          else if (index_used) 
          {
            seq_printf(sfile, "Xilinx Virtual Cable (XVC) associated with /dev/xil_xvc/cfg_ioc%d-%d\n", d, index_used);
            ++index_used;
          }
          else 
          {
            seq_printf(sfile, "Xilinx Virtual Cable (XVC) associated with /dev/xil_xvc/cfg_ioc%d\n", d);
            ++index_used;
          }

	}
	break;
      }
    }
    //End of add on for ADD_XVC
#endif
  }

  seq_printf(sfile, " \n");
  seq_printf(sfile, "The command 'echo <action> > /proc/wupper', executed as root,\n");
  seq_printf(sfile, "allows you to interact with the driver. Possible actions are:\n");
  seq_printf(sfile, "debug     -> Enable debugging\n");
  seq_printf(sfile, "nodebug   -> Disable debugging\n");
  seq_printf(sfile, "elog      -> Log errors to /var/log/message\n");
  seq_printf(sfile, "noelog    -> Do not log errors to /var/log/message\n");
  seq_printf(sfile, "swap      -> Enable automatic swapping of 0x7038 / 0x7039 and 0x427 / 0x428\n");
  seq_printf(sfile, "noswap    -> Disable automatic swapping of 0x7038 / 0x7039 and 0x427 / 0x428\n");
  seq_printf(sfile, "clearlock -> Clear all lock bits (Attention: Close processes that hold lock bits before you do this)\n");

  mutex_unlock(&procMutex);
  kdebug(("wupper(wupper_proc_show): mutex unlock OK\n"))

  return(0);
}

/********************************************************************************************************/
static ssize_t wupper_write_procmem(struct file *file, const char *buffer, size_t count, loff_t *startOffset)
/********************************************************************************************************/
{
  int len, loop, loop2;
  char textReceived[100];

  kdebug(("wupper(wupper_write_proc): robin_write_procmem called\n"))

  if (count > 99)
    len = 99;
  else
    len = count;

  if (copy_from_user(textReceived, buffer, len))
  {
    kerror(("wupper(wupper_write_proc): error from copy_from_user\n"))
    return(-EFAULT);
  }

  kdebug(("wupper(wupper_write_proc): len = %d\n", len))
  textReceived[len - 1] = '\0';
  kdebug(("wupper(wupper_write_proc): text passed = %s\n", textReceived))

  if (!strcmp(textReceived, "debug"))
  {
    debug = 1;
    kdebug(("wupper(wupper_write_proc): debugging enabled\n"))
  }

  if (!strcmp(textReceived, "nodebug"))
  {
    kdebug(("wupper(wupper_write_proc): debugging disabled\n"))
    debug = 0;
  }

  if (!strcmp(textReceived, "elog"))
  {
    kdebug(("wupper(wupper_write_proc): Error logging enabled\n"))
    errorlog = 1;
  }

  if (!strcmp(textReceived, "noelog"))
  {
    kdebug(("wupper(wupper_write_proc): Error logging disabled\n"))
    errorlog = 0;
  }
  
  if (!strcmp(textReceived, "swap"))
  {
    kdebug(("wupper(wupper_write_proc): Auto-swap enabled\n"))
    autoswap = 1;
  } 

  if (!strcmp(textReceived, "noswap"))
  {
    kdebug(("wupper(wupper_write_proc): Auto-swap disabled\n"))
    autoswap = 0;
  } 

  if (!strcmp(textReceived, "clearlock"))
  {
    kdebug(("wupper(wupper_write_proc): clearing all lock bits\n"))
    
    spin_lock_irqsave(&lock_lock, lock_irq_flags);   //Please do not disturb...
    
    for(loop = 0; loop < MAXCARDS; loop++)
    {
      global_locks[loop] = 0;
      for(loop2 = 0; loop2 < MAXLOCKBITS; loop2++)
      {
        lock_pid[loop][loop2] = 0;
        lock_tags[loop][loop2] = 0;
      }
    }

    lock_tag = 0;
    spin_unlock_irqrestore(&lock_lock, lock_irq_flags);   
	
    kdebug(("wupper(wupper_write_proc): lock bits hav been cleared\n"))
  } 
      
  return(len);
}


/*********************************************************/
int wupper_mmap(struct file *file, struct vm_area_struct *vma)
/*********************************************************/
{
  u32 moff, msize;

  // it should be "shared" memory
  if ((vma->vm_flags & VM_WRITE) && !(vma->vm_flags & VM_SHARED))
  {
    kerror(("wupper(wupper_mmap): writeable mappings must be shared, rejecting\n"))
    return(-EINVAL);
  }

  msize = vma->vm_end - vma->vm_start;
  moff = vma->vm_pgoff;
  kdebug(("wupper(wupper_mmap): offset: 0x%x, size: 0x%x\n", moff, msize))
  moff = moff << PAGE_SHIFT;
  if (moff & ~PAGE_MASK)
  {
    kerror(("wupper(wupper_mmap): offset not aligned: %u\n", moff))
    return(-EINVAL);
  }

#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0)
  vma->vm_flags |= VM_RESERVED;
  // we do not want to have this area swapped out, lock it
  vma->vm_flags |= VM_LOCKED;
#else
  #if LINUX_VERSION_CODE < KERNEL_VERSION(6,3,0)
  vma->vm_flags |= VM_DONTEXPAND;
  vma->vm_flags |= VM_DONTDUMP;
  // we do not want to have this area swapped out, lock it
  vma->vm_flags |= VM_LOCKED;
  #else
  vm_flags_set(vma, VM_DONTEXPAND);
  vm_flags_set(vma, VM_DONTDUMP);
  vm_flags_set(vma, VM_LOCKED);
  #endif
#endif

  if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, msize, vma->vm_page_prot) != 0)
  {
    kerror(("wupper(wupper_mmap): remap page range failed\n"))
    return(-EAGAIN);
  }

  vma->vm_ops = &wupper_vm_ops;
  return(0);
}


/******************************************/
void wupper_vmclose(struct vm_area_struct *vma)
/******************************************/
{
  kdebug(("wupper(wupper_mmap): closing mmap memory\n"))
}


/*************************************************************/
static long wupper_ioctl(struct file *file, u_int cmd, u_long arg)
/*************************************************************/
{
  card_params_t *deviceParams;
  static struct vm_area_struct *vmas, uvmas;
  u_int interrupt, lbit, device, address, inout;
  u_char capabilityId, capabilityIdOffset;
  u_short deviceControlRegister;
  u_int tlp, count;
  card_params_t temp;
  lock_params_t lockparams;

  kdebug(("wupper(wupper_ioctl): entered\n"))
  vmas = &uvmas;   //MJ: purpose unclear

  deviceParams = (card_params_t *)file->private_data;
  kdebug(("wupper(wupper_ioctl, pid=%d) device is %d\n", current->pid, deviceParams->slot))

  switch(cmd)
  {
  case GETCARDS:
    kdebug(("wupper(wupper_ioctl, GETCARDS\n"))
    if (copy_to_user(((int*)arg), &cdmap, sizeof(int) * 2 * MAXCARDS) != 0)
    {
      kerror(("wupper(wupper_ioctl, GETCARDS) Copy devicesFound to user space failed!\n"))
      return(-EFAULT); 
    }
    break;

  case GET_TLP:
    kdebug(("wupper(wupper_ioctl, GET_TLP)\n"))
    deviceParams = (card_params_t *)file->private_data;
    device = deviceParams->slot;

    // Offset of first capability list entry
    address = PCI_CAPABILITY_LIST;
    pci_read_config_byte(devices[device].pciDevice, address, &capabilityIdOffset);
    kdebug(("wupper(wupper_ioctl, GET_TLP) first capabilityIdOffset 0x%x\n", capabilityIdOffset))
    // Count protects against loop not terminating
    count = 0;
    while (count < PCI_CAP_ID_MAX)
    {
      pci_read_config_byte(devices[device].pciDevice, (u_int) capabilityIdOffset, &capabilityId);
      kdebug(("wupper(wupper_ioctl, GET_TLP) capabilityIdOffset 0x%x capabilityId 0x%x\n", capabilityIdOffset, capabilityId))
      if (capabilityId == PCI_CAP_ID_EXP)
        break;

      // Get next capability list entry offset
      address = (u_int) (capabilityIdOffset + PCI_CAP_LIST_NEXT);
      pci_read_config_byte(devices[device].pciDevice, address, &capabilityIdOffset);
      kdebug(("wupper(wupper_ioctl, GET_TLP) next capabilityIdOffset 0x%x\n", capabilityIdOffset))
      ++count;
    }
    if (count == PCI_CAP_ID_MAX)
    {
      kerror(("wupper(wupper_ioctl, GET_TLP) Did not find capability with TLP id\n"))
      return(-EINVAL);
    }

    address = capabilityIdOffset + PCI_EXP_DEVCTL;
    pci_read_config_word(devices[device].pciDevice, address, &deviceControlRegister);
    kdebug(("wupper(wupper_ioctl, GET_TLP) new deviceControlRegister 0x%x\n", deviceControlRegister))

    tlp = (deviceControlRegister & PCI_EXP_DEVCTL_PAYLOAD) >> 5;
    if (copy_to_user(((int*)arg), &tlp, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, GET_TLP) Copy value of TLP to user space failed!\n"))
      return(-EFAULT);
    }
    break;

  case WAIT_IRQ:    //WAIT_DMA
    kdebug(("wupper(wupper_ioctl, WAIT_IRQ, pid=%d) Entered\n", current->pid))
    deviceParams = (card_params_t *)file->private_data;
    device = deviceParams->slot;
    if (copy_from_user(&interrupt, (void *)arg, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, WAIT_IRQ) error from copy_from_user\n"))
      return(-EFAULT);
    }
    if (interrupt >= msixblock)
    {
      kerror(("wupper(wupper_ioctl, WAIT_IRQ) invalid interrupt specified %d\n", interrupt))
      return(-EINVAL);
    }
    kdebug(("wupper(wupper_ioctl, WAIT_IRQ, pid=%d) Waiting for interrupt %d\n", current->pid, interrupt))
    wait_event_interruptible(waitQueue, irqFlag[device][interrupt] == 1);
    //MJ: Rubini recommends to check the return code of wait_event_interruptible.
    //MJ: in case of a non-zero value -ERESTARTSYS should be returned

    irqFlag[device][interrupt] = 0;  //MJ: if other processes are waiting for the same interrupt, setting irqFlag[device][interrupt] back to 0 may hide the interrupt from them (see below)
    //MJ: According to Rubini (page 150, 151 of LDD3) there can be problems with race conditions if two processes are waiting for the same interrupt.
    //MJ: Is that a requirement for us?

    kdebug(("wupper(wupper_ioctl, WAIT_IRQ, pid=%d) finished waiting for IRQ %d\n", current->pid, interrupt))
    break;

  case CANCEL_IRQ_WAIT:
    kdebug(("wupper(wupper_ioctl) CANCEL_IRQ_WAIT\n"))
    deviceParams = (card_params_t *)file->private_data;
    device = deviceParams->slot;

    if (copy_from_user(&interrupt, (void *)arg, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, CANCEL_IRQ_WAIT) error from copy_from_user\n"))
      return(-EFAULT);
    }
    if (interrupt >= msixblock)
    {
      kerror(("wupper(wupper_ioctl, CANCEL_IRQ_WAIT) invalid interrupt specified %d\n", interrupt))
      return(-EINVAL);
    }

    kdebug(("wupper(wupper_ioctl, CANCEL_IRQ_WAIT) Cancelling interrupt %d\n", interrupt))
    // set flag to 1, wake_up_interruptible will wake up a process if the flag specified in wait_event_interruptible is set to 1
    irqFlag[device][interrupt] = 1;          //MJ: The access to the interrupt related data structures seems to be vulnerable to race conditions (two processes callinh different ioctls ate the same time)

    // Wake up everybody who was waiting for an interrupt 
    wake_up_interruptible(&waitQueue);   //MJ: Woken up process will not know that the interrupt has not arrived.
    break;

   case CLEAR_IRQ:
     kdebug(("wupper(wupper_ioctl, CLEAR_IRQ)\n"))
     deviceParams = (card_params_t *)file->private_data;
     device = deviceParams->slot;

     if (copy_from_user(&interrupt, (void *)arg, sizeof(u_int)) != 0)
     {
       kerror(("wupper(wupper_ioctl, CLEAR_IRQ) error from copy_from_user\n"))
       return(-EFAULT);
     }
     if (interrupt >= msixblock)
     {
       kerror(("wupper(wupper_ioctl, CLEAR_IRQ) invalid interrupt specified %d\n", interrupt))
       return(-EINVAL);
     }

     kdebug(("wupper(wupper_ioctl, CLEAR_IRQ) Clearing interrupt %d\n", interrupt))

     // set flag to 0 to clear a potentially pending (unsolicited) interrupt. 
     irqFlag[device][interrupt] = 0;          //MJ: The access to the interrupt related data structures seems to be vulnerable to race conditions (two processes callinh different ioctls ate the same time)
     break;
	
  case RESET_IRQ_COUNTERS:
    kdebug(("wupper(wupper_ioctl, RESET_IRQ_COUNTERS)\n"))
    deviceParams = (card_params_t *)file->private_data;
    device = deviceParams->slot;

    if (copy_from_user(&interrupt, (void *)arg, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, RESET_IRQ_COUNTERS) error from copy_from_user\n"))
      return(-EFAULT);
    }
    if (interrupt >= msixblock)
    {
      kerror(("wupper(wupper_ioctl, RESET_IRQ_COUNTERS) invalid interrupt specified %d\n", interrupt))
      return(-EINVAL);
    }	

    kdebug(("wupper(wupper_ioctl, RESET_IRQ_COUNTERS) Resetting counters of interrupt %d\n", interrupt))	
    irqCount[device][interrupt] = 0;

    break;

  case MASK_IRQ:   
    deviceParams = (card_params_t *)file->private_data;
    kdebug(("wupper(wupper_ioctl, MASK_IRQ, pid=%d) called for device %d\n", current->pid, deviceParams->slot))
    device = deviceParams->slot;

    if (copy_from_user(&interrupt, (void *)arg, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, MASK_IRQ, pid=%d) error from copy_from_user\n", current->pid))
      return(-EFAULT);
    }
    if (interrupt >= msixblock)
    {
      kerror(("wupper(wupper_ioctl, MASK_IRQ, pid=%d) invalid interrupt specified %d\n", current->pid, interrupt))
      return(-EINVAL);
    }
    // check that interrupt was not already masked
    if (irqMasked[device][interrupt] == 0)
    {
      disable_irq(msixTable[device][interrupt].vector);
      irqMasked[device][interrupt] = 1;
      kdebug(("wupper(wupper_ioctl, MASK_IRQ, pid=%d) masked interrupt %d\n", current->pid, interrupt))
    }
    else
      kdebug(("wupper(wupper_ioctl, MASK_IRQ, pid=%d) interrupt %d already masked -> no action\n", current->pid, interrupt))

    break;

  case UNMASK_IRQ:
    deviceParams = (card_params_t *)file->private_data;
    kdebug(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) called for device %d\n", current->pid, deviceParams->slot))
    device = deviceParams->slot;
    if (copy_from_user(&interrupt, (void *)arg, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) error from copy_from_user\n", current->pid))
      return(-EFAULT);
    }

    if (interrupt >= msixblock)
    {
      kerror(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) invalid interrupt specified %d\n", current->pid, interrupt))
      return(-EINVAL);
    }

    // check that interrupt was not already unmasked
    if (irqMasked[device][interrupt] == 1)
    {
      kdebug(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) msixTable[%d][%d].vector = %d\n", current->pid, device, interrupt, msixTable[device][interrupt].vector))

      enable_irq(msixTable[device][interrupt].vector);
      irqMasked[device][interrupt] = 0;
      kdebug(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) unmasked interrupt %d\n", current->pid, interrupt))
    }
    else
    {
      kdebug(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) msixTable[%d][%d].vector = %d\n", current->pid, device, interrupt, msixTable[device][interrupt].vector))
      kdebug(("wupper(wupper_ioctl, UNMASK_IRQ, pid=%d) interrupt %d already unmasked -> no action\n", current->pid, interrupt))
    }

    break;

  case SETCARD:
    if (copy_from_user( (void *) &temp, (void *)arg, sizeof(card_params_t)) != 0)
    {
      kerror(("wupper(wupper_ioctl, SETCARD) error from copy_from_user\n"))
      return(-EFAULT);
    }

    deviceParams = (card_params_t *)file->private_data;
    device = temp.slot;
    kdebug(("wupper(wupper_ioctl, SETCARD) device = %d\n", device))

    if (device >= MAXCARDS)
    {
      kerror(("wupper(wupper_ioctl, SETCARD) Invalid (%d) slot number\n", device))
      return(-EINVAL);
    }
    if (devices[device].pciDevice == NULL)
    {
      kerror(("wupper(wupper_ioctl, SETCARD) No device at this (%d) slot!\n", device))
      return(-EINVAL);
    }

    kdebug(("wupper(wupper_ioctl, SETCARD) lock_mask = 0x%08x\n", temp.lock_mask))

    if (temp.lock_mask)  
    {
      spin_lock_irqsave(&lock_lock, lock_irq_flags);   //Please do not disturb...

      //Check if the bits requested by this process are already locked
      //Using global_locks is just for convenience. We could also search lock_pid bit by bit

      kdebug(("wupper(wupper_ioctl, SETCARD) Old global_locks[%d] is 0x%08x\n", temp.slot, global_locks[temp.slot]))
      kdebug(("wupper(wupper_ioctl, SETCARD) user lock_mask is 0x%08x\n", temp.lock_mask))

      temp.lock_error = 0;
      if (temp.lock_mask & global_locks[temp.slot])
      {
        kerror(("wupper(wupper_ioctl, SETCARD) ERROR(locking conflict): global_locks[%d] is 0x%08x and user lock_mask is 0x%08x\n", temp.slot, global_locks[temp.slot], temp.lock_mask))
        temp.lock_error = global_locks[temp.slot];
	devices[device].lock_error = temp.lock_error;
      }	  
      else
      {
        //No conflict. Remember the newly locked bits
	global_locks[temp.slot] |= temp.lock_mask;
        kdebug(("wupper(wupper_ioctl, SETCARD) New global_locks[%d] is 0x%08x\n", temp.slot, global_locks[temp.slot]))

	//Update the table for proc_read
	for(lbit = 0; lbit < MAXLOCKBITS; lbit++)
	{
	  if (temp.lock_mask & (1 << lbit))
	  {
            kdebug(("wupper(wupper_ioctl, SETCARD) registering bit %d of device %d for PID %d and tag %d\n", lbit, temp.slot, current->pid, lock_tag))
	    lock_pid[temp.slot][lbit] = current->pid;
	    lock_tags[temp.slot][lbit] = lock_tag;
	  }
	}
        deviceParams->lock_tag = lock_tag;
	devices[device].lock_tag = lock_tag;
	devices[device].lock_mask = temp.lock_mask;
	devices[device].lock_error = temp.lock_error;
	lock_tag++;
      }
      spin_unlock_irqrestore(&lock_lock, lock_irq_flags);   
    }
    else
    {
      kdebug(("wupper(wupper_ioctl, SETCARD) No locking requested\n"))
      deviceParams->lock_tag = lock_tag;
      devices[device].lock_tag = 0;
      devices[device].lock_mask = 0;
      devices[device].lock_error = 0;
    }

    deviceParams->slot = device;
    deviceParams->baseAddressBAR0 = devices[device].baseAddressBAR0;
    deviceParams->sizeBAR0 = devices[device].sizeBAR0;
    deviceParams->baseAddressBAR1 = devices[device].baseAddressBAR1;
    deviceParams->sizeBAR1 = devices[device].sizeBAR1;
    deviceParams->baseAddressBAR2 = devices[device].baseAddressBAR2;
    deviceParams->sizeBAR2 = devices[device].sizeBAR2;
    // OK, we have a valid slot, copy configuration back to user

    kdebug(("wupper(wupper_ioctl, SETCARD) devices[device].lock_error before copy_to_user is %d\n", devices[device].lock_error))
    kdebug(("wupper(wupper_ioctl, SETCARD) sizeof(card_params_t) is %ld\n", sizeof(card_params_t)))

    if (copy_to_user(((card_params_t *)arg), &devices[device], sizeof(card_params_t)) != 0)
    {
      kerror(("wupper(wupper_ioctl, SETCARD) Copy card_params_t to user space failed!\n"))
      return(-EFAULT);
    }
    kdebug(("wupper(wupper_ioctl, SETCARD) end of ioctl SETCARD\n"))
    break;

  case GETLOCK:
    if (copy_from_user((void *) &inout, (void *)arg, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, GETLOCK) error from copy_from_user\n"))
      return(-EFAULT);
    }

    kdebug(("wupper(wupper_ioctl, GETLOCK) inout = %d\n", inout))
    
    if (inout >= MAXCARDS)
    {
      kerror(("wupper(wupper_ioctl, GETLOCK) Invalid slot number %d\n", inout))
      return(-EINVAL);
    }

    address = inout;                  //just to avoid using "inout" for two different purposes
    inout = global_locks[address];    

    kdebug(("wupper(wupper_ioctl, GETLOCK) global_locks[%d] = 0x%08x\n", address, global_locks[address]))
    kdebug(("wupper(wupper_ioctl, GETLOCK) inout = 0x%08x\n", inout))

    if (copy_to_user(((u_int *)arg), &inout, sizeof(u_int)) != 0)
    {
      kerror(("wupper(wupper_ioctl, GETLOCK) Copy card_params_t to user space failed!\n"))
      return(-EFAULT);
    }
    kdebug(("wupper(wupper_ioctl, GETLOCK) end of ioctl GETLOCK\n"))   
    break;

	
    //MJ Note: technically the garbage collection in wupper_Release is able to return the resource locks. Therefore the RELEASELOCK ioctl
    //         is luxury and can be removed unless an explicit release of the resources is desired.
    //MJ: The explicit release have the (useful?) advantage that a process can close access to a WUPPER device and re-open it with 
    //    different locking parameters without a restart of that process. 
    	
  case RELEASELOCK:
    kdebug(("wupper(wupper_ioctl, RELEASELOCK): called\n"))
    if (copy_from_user((void *) &lockparams, (void *)arg, sizeof(lock_params_t)) != 0)
    {
      kerror(("wupper(wupper_ioctl, RELEASELOCK) error from copy_from_user\n"))
      return(-EFAULT);
    }
    kdebug(("wupper(wupper_ioctl, RELEASELOCK): called for PID = %d and lockparams.slot = %d and lockparams.lock_tag = %d \n", current->pid, lockparams.slot, lockparams.lock_tag))

    spin_lock_irqsave(&lock_lock, lock_irq_flags);   //Please do not disturb...

    kdebug(("wupper(wupper_ioctl, RELEASELOCK) Old global_locks[%d] is 0x%08x\n", lockparams.slot, global_locks[lockparams.slot]))
    for(lbit = 0; lbit < MAXLOCKBITS; lbit++)
    {
      if (lock_pid[lockparams.slot][lbit] == current->pid && lock_tags[lockparams.slot][lbit] == lockparams.lock_tag)
      {
        kdebug(("wupper(wupper_ioctl, RELEASELOCK) unregistering bit %d of device %d for PID %d\n", lbit, lockparams.slot, current->pid))
	lock_pid[lockparams.slot][lbit] = 0;
	lock_tags[lockparams.slot][lbit] = 0;
        global_locks[lockparams.slot] = global_locks[lockparams.slot] & ~(1 << lbit);
      }
    }
    kdebug(("wupper(wupper_ioctl, RELEASELOCK) New global_locks[%d] is 0x%08x\n", lockparams.slot, global_locks[lockparams.slot]))
    spin_unlock_irqrestore(&lock_lock, lock_irq_flags);   
    break;

  default:
    kerror(("wupper(wupper_ioctl, default) Unknown ioctl 0x%x\n", cmd))
    return(-EINVAL);
    }

  return 0;
}



#ifdef ADD_XVC
/**
 * The section below adds Xilinx Virtual Cable (XVC) support. 
 * It was copied from the Xilinx XVC cable driver by Frans Schreuder
 * The section below will be maintained by Frans Schreuder
 */

static int xil_xvc_init(void)
{
   
  kdebug(("wupper(xil_xvc_init): Starting...\n"))

  if (!xil_xvc_devices) {
    xil_xvc_devices = (struct xil_xvc_char *) kmalloc(sizeof(struct xil_xvc_char) * CHAR_DEVICES_MAX, GFP_KERNEL);
  }

  // Register the character packet device major and minor numbers
  kdebug(("wupper(xil_xvc_init): alloc_chrdev_region\n"))
  xvc_ioc_dev_region_status = alloc_chrdev_region(&xvc_ioc_dev_region, 0, CHAR_DEVICES_MAX, "xilinx_xvc_pci_ioc_driver_region");
  if (xvc_ioc_dev_region_status != 0) {
    kerror(("wupper(xil_xvc_init): xvc_ioc_dev_region_status != 0\n"))
    xil_xvc_cleanup();
    return xvc_ioc_dev_region_status;
  }
  // Add the character device, no actual files yet
  kdebug(("wupper(xil_xvc_init): cdev_init and cdev_add\n"))
  cdev_init(&xvc_char_ioc_dev, &xil_xvc_ioc_ops);
  xvc_char_ioc_dev.owner = THIS_MODULE;
  cdev_add(&xvc_char_ioc_dev, xvc_ioc_dev_region, CHAR_DEVICES_MAX);

  // Register the character device class for the actual files
  kdebug(("wupper(xil_xvc_init): class create\n"))
#if LINUX_VERSION_CODE < KERNEL_VERSION(6,3,0)
  #ifdef  RHEL_RELEASE_CODE
    #if RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(9,4)
  xvc_dev_class = class_create(THIS_MODULE, "xil_xvc_class");
    #else
  xvc_dev_class = class_create("xil_xvc_class");
    #endif
  #else
  xvc_dev_class = class_create(THIS_MODULE, "xil_xvc_class");
  #endif
#else
  xvc_dev_class = class_create("xil_xvc_class");
#endif
  if (IS_ERR(xvc_dev_class)) {
    kerror(("wupper(xil_xvc_init): IS_ERR(xvc_dev_class)\n"))
    xil_xvc_cleanup();
    return PTR_ERR(xvc_dev_class);
  }

  // init device list
  kdebug(("wupper(xil_xvc_init): memset\n"))
  memset(xil_xvc_devices, 0, sizeof(*xil_xvc_devices));

  return 0;
}


static int xil_xvc_probe(struct pci_dev *dev)
{
  if (dev->device == PCI_DEVICE_ID_CERN_FW1) //In the first endpoint we can look for an XVC virtual JTAG cable in configuration space.
  {
    //FS: Some variables needed for XVC Xilinx Virtual Cable.
    dev_t ioc_device_number;
    char ioc_device_name[25];
    struct device *xvc_ioc_device;
    int status;
    int d = 0;
    int u = 0;
    for (d = 0; d < CHAR_DEVICES_MAX; ++d) 
    {
      if (xil_xvc_devices[d * USER_CONFIG_COUNT].pci_dev == NULL) 
      {
        for (u = 0; u < USER_CONFIG_COUNT; ++u) 
        {
          int i = d * USER_CONFIG_COUNT + u;
          const char *name;
          int index_used = 0;

          // Add device to table
          xil_xvc_devices[i].pci_dev = dev;
          memset(&xil_xvc_devices[i].xvc_algo, 0, sizeof(struct xvc_algo_t));

          status = setup_xvc_algo(xil_xvc_devices + i, user_configs + u);
          if (status != 0) 
          {
            kerror(("wupper(xil_xvc_probe): Failed to setup xvc capability\n"))
            xil_xvc_devices[i].pci_dev = NULL;
            return status;
          }
          xil_xvc_devices[i].user_config = (struct pcie_user_config *) user_configs + u;

          // Character driver file
          ioc_device_number = get_device_number(xvc_ioc_dev_region, i);

          // Make device character file name with right number
          name = user_configs[u].name;
          if (name && name[0]) 
          {
            sprintf(ioc_device_name, "xil_xvc/cfg_ioc%d_%s", d, name);
          }
          else if (index_used) 
          {
            sprintf(ioc_device_name, "xil_xvc/cfg_ioc%d-%d", d, index_used);
            ++index_used;
          }
          else 
          {
            sprintf(ioc_device_name, "xil_xvc/cfg_ioc%d", d);
            ++index_used;
          }

          xvc_ioc_device = device_create(xvc_dev_class, NULL, ioc_device_number, NULL, ioc_device_name);
          if (IS_ERR(xvc_ioc_device)) 
          {
            kerror(("wupper(xil_xvc_probe): Failed to create the device %d %s\n", i, ioc_device_name))
          } 
          else 
          {
            kerror(("wupper(xil_xvc_probe): Created device %d %s, reg offset (0x%lX)\n", i, ioc_device_name, xil_xvc_devices[i].xvc_algo.offset.cfg))
          }
        }
        break;
      }
    }
  }
  return 0;
}

static void xil_xvc_remove(struct pci_dev *dev)
{  
   int d,u;
   for (d = 0; d < CFG_DEVICES_MAX; ++d) {
    if (dev != NULL && xil_xvc_devices[d * USER_CONFIG_COUNT].pci_dev == dev) {
      for (u = 0; u < USER_CONFIG_COUNT; ++u) {
        int i = d * USER_CONFIG_COUNT + u;
        
        xil_xvc_devices[i].pci_dev = NULL;

        device_destroy(xvc_dev_class, get_device_number(xvc_ioc_dev_region, i));
        kerror(("wupper(xil_xvc_remove): XVC removal found at %d\n", i))
      }
      break;
    }
  }
}


static void xil_xvc_cleanup(void) {
    
  kdebug(("wupper(xil_xvc_cleanup): Cleaning up resources...\n"))
 
  if (!IS_ERR(xvc_dev_class)) {
    kdebug(("wupper(xil_xvc_cleanup): class_unregister: removed to prevent class_destroy to generate an error upon first rmmod\n"))
    // class_unregister(xvc_dev_class);
  }
  if (!IS_ERR(xvc_dev_class)) {
      kdebug(("wupper(xil_xvc_cleanup): class_destroy\n"))
      class_destroy(xvc_dev_class);
  }

  if (xvc_char_ioc_dev.owner != NULL) {
      kdebug(("wupper(xil_xvc_cleanup): cdev_del\n"))
      cdev_del(&xvc_char_ioc_dev);
  }

  if (xvc_ioc_dev_region_status == 0) {
    kdebug(("wupper(xil_xvc_cleanup): unregister_chrdev_region\n"))
    unregister_chrdev_region(xvc_ioc_dev_region, CFG_DEVICES_MAX);
  }

  kdebug(("wupper(xil_xvc_cleanup): kfree(xil_xvc_devices)\n"))
  if (xil_xvc_devices) kfree(xil_xvc_devices);
}


static long setup_xvc_algo(struct xil_xvc_char *xvc_char, const struct pcie_user_config *user_config) {
  int status = -EINVAL;
  int bar_index = user_config->bar_info.bar_index;
  kdebug(("wupper(setup_xvc_algo): entering setup_xvc_algo...\n"))
  // initialize algo selection if not already selected
  if (!xvc_char->xvc_algo.type) {
    
    struct xvc_algo_t *algo = &xvc_char->xvc_algo;

    // try to find the cfg capability if CONFIG or AUTO was specified by user
    if (user_config->config_space == CONFIG || user_config->config_space == AUTO) {
      status = xil_xvc_get_offset(
        xvc_char->pci_dev, 
        user_config->config_info.config_vsec_id, 
        user_config->config_info.config_vsec_rev, 
        &algo->offset.cfg);
      kerror(("wupper(setup_xvc_algo): setup_xvc_algo status: %i\n", status))
      if (status == 0) {
        // found the capability so use CFG space;
        algo->type = XVC_ALGO_CFG;
        return status;
      }
    }
    // try to use BAR space if selected, or fall through for AUTO
    if (user_config->config_space == BAR || user_config->config_space == AUTO) {
      resource_size_t bar_start;
      resource_size_t bar_len;
      resource_size_t map_len;

      if ((!pci_resource_flags(xvc_char->pci_dev, bar_index)) & IORESOURCE_MEM) {
        kerror(("wupper(setup_xvc_algo): Incorrect BAR configuration\n"))
        return -ENODEV;
      }

      bar_start = pci_resource_start(xvc_char->pci_dev, bar_index);
      bar_len = pci_resource_len(xvc_char->pci_dev, bar_index);
      map_len = bar_len;
        
      if (!bar_len) {
        kerror(("wupper(setup_xvc_algo): BAR #%d is not present.\n", bar_index))
      } else {
        // add user specified BAR offset to base address of mapping
        algo->offset.bar = pci_iomap(xvc_char->pci_dev, bar_index, map_len) + user_config->bar_info.bar_offset;
        if (!(algo->offset.bar)) {
          kerror(("wupper(setup_xvc_algo): Could not map BAR %d\n", bar_index))
          return -ENODEV;
        } else {
          algo->type = XVC_ALGO_BAR;
          kerror(("wupper(setup_xvc_algo): BAR%d at 0x%llx mapped at 0x%p, length=%llu(/%llu)\n", bar_index, (u64)bar_start, algo->offset.bar, (u64)map_len, (u64)bar_len))
          status = 0;
        }
      }
    }
  }

  return status;
}

long char_ctrl_ioctl(struct file *file_p, unsigned int cmd, unsigned long arg) {
  int char_index = iminor(file_p->f_path.dentry->d_inode) - MINOR(xvc_ioc_dev_region);
  int status = 0;
  unsigned long pci_config_lock_flags = 0;

  if (xil_xvc_devices[char_index].pci_dev == NULL) {
    kerror(("wupper(char_ctrl_ioctl): Could not find char_index %d\n", char_index))
    return -EFAULT;
  }
  
  spin_lock_irqsave(&file_p->f_path.dentry->d_inode->i_lock, pci_config_lock_flags);

  switch (cmd)
  {
  case XDMA_IOCXVC:
    status = xil_xvc_ioctl(xil_xvc_devices[char_index].pci_dev, &xil_xvc_devices[char_index].xvc_algo, (void __user *)arg);
    break;
  case XDMA_RDXVC_PROPS:
    status = xil_xvc_readprops(xil_xvc_devices[char_index].pci_dev, &xil_xvc_devices[char_index].xvc_algo, xil_xvc_devices[char_index].user_config, (void __user *)arg);
    break;
  default:
    kerror(("wupper(char_ctrl_ioctl): Will return -ENOIOCTLCMD\n"))
    status = -ENOIOCTLCMD;
    break;
  }
  
  spin_unlock_irqrestore(&file_p->f_path.dentry->d_inode->i_lock, pci_config_lock_flags);

  return status;
}


int pci_read32(struct pci_dev *dev, struct xvc_algo_t *algo, int offset, u32 *value) {

  switch(algo->type)
  {
  case XVC_ALGO_BAR:
    *value = ioread32(algo->offset.bar + offset);
    return 0;
  case XVC_ALGO_CFG:
    return pci_read_config_dword(dev, algo->offset.cfg + offset, value);
  default:;
  }
  
  kerror(("wupper(pci_read32): You should not be here\n"))
  return -1;
}

int pci_write32(struct pci_dev *dev, struct xvc_algo_t *algo, int offset, u32 value) {

  switch(algo->type)
  {
  case XVC_ALGO_BAR:
    iowrite32(value, algo->offset.bar + offset);
    return (0);
  case XVC_ALGO_CFG:    
    return pci_write_config_dword(dev, algo->offset.cfg + offset, value);
  default:;
  }

  kerror(("wupper(pci_write32): You should not be here\n"))
  return -1;
}

static int xil_xvc_shift_bits(struct pci_dev *pci_dev, struct xvc_algo_t *algo, u32 tms_bits, u32 tdi_bits, u32 *tdo_bits) {
  int status;
  u32 control_reg_data;
  u32 write_reg_data;
  int count = 100;

  // Set tms bits
  status = pci_write32(pci_dev, algo, TMS_REG_OFFSET, tms_bits);
  if (status != 0) {
    kerror(("wupper(xil_xvc_shift_bits): error code %d received from pci_write32\n", status))
    return status;
  }

  // Set tdi bits and shift data out
  status = pci_write32(pci_dev, algo, TDI_REG_OFFSET, tdi_bits);
  if (status != 0) {
    kerror(("wupper(xil_xvc_shift_bits): error code %d received from pci_write32\n", status))
    return status;
  }

  // poll status to wait for completion
  if (algo->type == XVC_ALGO_BAR)  {

    // Read control register
    status = pci_read32(pci_dev, algo, CONTROL_REG_OFFSET, &control_reg_data);
    if (status != 0) {
      kerror(("wupper(xil_xvc_shift_bits): error code %d received from pci_read32\n", status))
      return status;
    }

    // Enable shift operation in control register
    write_reg_data = control_reg_data | 0x01;

    // Write control register
    status = pci_write32(pci_dev, algo, CONTROL_REG_OFFSET, write_reg_data);
    if (status != 0) {
      kerror(("wupper(xil_xvc_shift_bits): error code %d received from pci_write32\n", status))
      return status;
    }

    while (count) {
      // Read control reg to check shift operation completion
      status = pci_read32(pci_dev, algo, CONTROL_REG_OFFSET, &control_reg_data);
      if (status != 0) {
        kerror(("wupper(xil_xvc_shift_bits): error code %d received from pci_read32\n", status))
        return status;
      }
      if ((control_reg_data & 0x01) == 0)  {
        break;
      }
      count--;
    }
    if (count == 0)  {
      kerror(("wupper(xil_xvc_shift_bits): XVC bar transaction timed out (%0X)\n", control_reg_data))
      return -ETIMEDOUT;
    }
  }

  // Read tdo bits back out
  status = pci_read32(pci_dev, algo, TDO_REG_OFFSET, tdo_bits);
  if (status != 0) {
    kerror(("wupper(xil_xvc_shift_bits): error code %d received from pci_read32\n", status))
  }
      
  return status;
}

ssize_t xil_xvc_ioctl(struct pci_dev *pci_dev, struct xvc_algo_t *algo, const char __user *arg) {
  struct xil_xvc_ioc xvc_obj;
  u32 operation_code;
  u32 num_bits;
  int num_bytes;
  char *tms_buf_temp = NULL;
  char *tdi_buf_temp = NULL;
  char *tdo_buf_temp = NULL;
  int current_bit;
  u32 bypass_status;
  int status = 0;

  if (pci_dev == NULL || algo == NULL || algo->type == XVC_ALGO_NULL) {
    kerror(("wupper(xil_xvc_ioctl): Will return -EINVAL\n"))
    return -EINVAL;
  }

  if ((status = copy_from_user((void *)&xvc_obj, arg, sizeof(struct xil_xvc_ioc)))) {
    goto cleanup;
  }

  operation_code = xvc_obj.opcode;

  // Invalid operation type, no operation performed
  if (operation_code != 0x01 && operation_code != 0x02) {
    kerror(("wupper(xil_xvc_ioctl): Will return 0\n"))
    return 0;
  }

  num_bits = xvc_obj.length;
  num_bytes = (num_bits + 7) / 8;

  // Allocate and copy data into temporary buffers
  tms_buf_temp = (char*) kmalloc(num_bytes, GFP_KERNEL);
  if (tms_buf_temp == NULL) {
    status = -ENOMEM;
    goto cleanup;
  }
  if ((status = copy_from_user((void *)tms_buf_temp, xvc_obj.tms_buf, num_bytes))) {
    goto cleanup;
  }

  tdi_buf_temp = (char*) kmalloc(num_bytes, GFP_KERNEL);
  if (tdi_buf_temp == NULL) {
    status = -ENOMEM;
    goto cleanup;
  }
  if ((status = copy_from_user((void *)tdi_buf_temp, xvc_obj.tdi_buf, num_bytes))) {
    goto cleanup;
  }

  // Allocate TDO buffer
  tdo_buf_temp = (char*) kmalloc(num_bytes, GFP_KERNEL);
  if (tdo_buf_temp == NULL) {
    status = -ENOMEM;
    goto cleanup;
  }


  if (operation_code == 0x2) {
    bypass_status = 0x2;
  } else {
    bypass_status = 0x0;
  }

  status = pci_write32(pci_dev, algo, CONTROL_REG_OFFSET, bypass_status);
  if (status) {
    goto cleanup;
  }

  // Set length register to 32 initially if more than one word-transaction is to be done
  if (num_bits >= 32) {
    status = pci_write32(pci_dev, algo, LENGTH_REG_OFFSET, 0x20);
    if (status) {
      goto cleanup;
    }
  }

  current_bit = 0;
  while (current_bit < num_bits) {
    int shift_num_bytes;
    int shift_num_bits = 32;

    u32 tms_store = 0;
    u32 tdi_store = 0;
    u32 tdo_store = 0;

    if (num_bits - current_bit < shift_num_bits) {
      shift_num_bits = num_bits - current_bit;
      // do LENGTH_REG_OFFSET here
      // Set number of bits to shift out
      status = pci_write32(pci_dev, algo, LENGTH_REG_OFFSET, shift_num_bits);
      if (status != 0) {
        goto cleanup;
      }
    }

    // Copy only the remaining number of bytes out of user-space
    shift_num_bytes = (shift_num_bits + 7) / 8;
    
    memcpy(&tms_store, tms_buf_temp + (current_bit / 8), shift_num_bytes);
    memcpy(&tdi_store, tdi_buf_temp + (current_bit / 8), shift_num_bytes);

    // Shift data out and copy to output buffer
    status = xil_xvc_shift_bits(pci_dev, algo, tms_store, tdi_store, &tdo_store);
    if (status) {
      goto cleanup;
    }

    memcpy(tdo_buf_temp + (current_bit / 8), &tdo_store, shift_num_bytes);

    current_bit += shift_num_bits;
  }

  if (copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf_temp, num_bytes)) {
    status = -EFAULT;
    goto cleanup;
  }

cleanup:
  if (tms_buf_temp) kfree(tms_buf_temp);
  if (tdi_buf_temp) kfree(tdi_buf_temp);
  if (tdo_buf_temp) kfree(tdo_buf_temp);
  kerror(("wupper(xil_xvc_ioctl): status = %d\n", status))
  return status;
}

ssize_t xil_xvc_readprops(struct pci_dev *pci_dev, struct xvc_algo_t *algo, struct pcie_user_config *user_config, const char __user *arg) {
  struct xil_xvc_properties xvc_props_obj;
  int status = 0;

  if (pci_dev == NULL || algo == NULL || user_config == NULL || algo->type == XVC_ALGO_NULL) {
    return -EINVAL;
  }

  xvc_props_obj.xvc_algo_type   = (unsigned int) algo->type;
  xvc_props_obj.config_vsec_id  = user_config->config_info.config_vsec_id;
  xvc_props_obj.config_vsec_rev = user_config->config_info.config_vsec_rev;
  xvc_props_obj.bar_index     = user_config->bar_info.bar_index;
  xvc_props_obj.bar_offset    = user_config->bar_info.bar_offset;
  
  if (copy_to_user((void *)arg, &xvc_props_obj, sizeof(xvc_props_obj))) {
    kerror(("wupper(xil_xvc_readprops): can't copy to user\n"))
    status = -ENOMEM;
    goto cleanup;
  }

cleanup:
  return status;
}

int xil_xvc_get_offset(struct pci_dev *pci_dev, unsigned user_vsec_id, unsigned user_vsec_rev, size_t *offset) {
  int status;
  size_t i = 0x100; // starting offset for extended capability registers
  struct xvc_algo_t algo = {
    .type = XVC_ALGO_CFG, 
    .offset.cfg = 0
  };
    kdebug(("wupper(xil_xvc_get_offset): xil_xvc_get_offset...\n"))
  while (VALID_OFFSET(i))
  {
    unsigned word = 0x0;
    unsigned vsec_id;
    unsigned vsec_rev;

    // read vendor-specific header
    status = pci_read32(pci_dev, &algo, i + 4, &word);
    if (status != 0) {
      kerror(("wupper(xil_xvc_get_offset): status = %d\n", status))
      return status;
    }

    vsec_id = word & VSEC_ID_MASK;
    vsec_rev = (word & VSEC_REV_MASK) >> VSEC_REV_SHIFT;

    if (vsec_id == user_vsec_id && vsec_rev == user_vsec_rev)
    {
          break;
    }

    // read next offset
    status = pci_read32(pci_dev, &algo, i, &word);
    if (status != 0) {
      kerror(("wupper(xil_xvc_get_offset): status = %d\n", status))
      return status;
    }

    i = word >> NEXT_CAP_SHIFT;
  }

  if (VALID_OFFSET(i))
  {
    // found capability
    if (offset) *offset = i;
  }
  else
  {
    status = -ENXIO;
  }

  return status;
}
#endif
